/*
 *	File: BaseParser.h
 *	Version: 1.0
 *	Desc:
 *	History:
 *	1. 2011-09-12, Created, zwx
 */

#ifndef BASEPARSER_H_
#define BASEPARSER_H_
using namespace std;
#include "../Entity/Document.h"
#include "../Helper/StringHelper.h"

class BaseParser {
public:
	StringHelper sh;
	BaseParser();
	virtual ~BaseParser();
	void parse(Document* doc, char* text);
	void tokenize();
	void normalize(string token, Document* doc, size_t token_offset_in_doc);
	bool isStopWord(string token);
};

#endif /* BASEPARSER_H_ */
